{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8fb91fb1-d800-4fac-81c4-06c2a2636672",
   "metadata": {},
   "source": [
    "模型构造"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9110542f-da89-49b4-8b07-165387e03ad5",
   "metadata": {},
   "source": [
    "继承 MODULE 类来构造模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "17b18109-15ce-46a8-9411-d97dabcdfcd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "class MLP(nn.Module):\n",
    "    # 声明带有模型参数的层，这⾥声明了两个全连接层\n",
    "    def __init__(self, **kwargs):\n",
    "        # 调⽤MLP⽗类Block的构造函数来进⾏必要的初始化。这样在构造实例时还可以指定其他函数\n",
    "        # 参数，如“模型参数的访问、初始化和共享”⼀节将介绍的模型参数params\n",
    "        super(MLP, self).__init__(**kwargs)\n",
    "        self.hidden = nn.Linear(784, 256) # 隐藏层\n",
    "        self.act = nn.ReLU()\n",
    "        self.output = nn.Linear(256, 10) # 输出层\n",
    "        \n",
    "        # 定义模型的前向计算，即如何根据输⼊x计算返回所需要的模型输出\n",
    "        \n",
    "    def forward(self, x):\n",
    "        a = self.act(self.hidden(x))\n",
    "        return self.output(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "65fa3ce3-a13b-4f9b-8e4a-ccf319cb5493",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP(\n",
      "  (hidden): Linear(in_features=784, out_features=256, bias=True)\n",
      "  (act): ReLU()\n",
      "  (output): Linear(in_features=256, out_features=10, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0240, -0.1623, -0.1567,  0.0950,  0.0851, -0.0473, -0.1164,  0.0105,\n",
       "         -0.2705,  0.0449],\n",
       "        [-0.0953, -0.1693,  0.1131,  0.2009,  0.2001, -0.0748, -0.1075, -0.1609,\n",
       "         -0.1203,  0.0915]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = torch.rand(2, 784)\n",
    "#实例化得到net模型\n",
    "net = MLP()\n",
    "print(net)\n",
    "#输入数据X做一次前向计算\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c11d4c1-3af0-4426-9ae8-27d3bd38c755",
   "metadata": {},
   "source": [
    "MODULE 的⼦类\n",
    "PyTorch还实现了继承⾃ Module 的可以⽅便构建模型的类: 如 Sequential 、 ModuleList 和 ModuleDict 等等"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "535887e5-2bba-4ba4-9648-c0776f277933",
   "metadata": {},
   "source": [
    "当模型的前向计算为简单串联各个层的计算时， Sequential 类可以通过更加简单的⽅式定义模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d8aac3ee-8915-448b-800a-bae2f7e8c3e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#继承自 torch.nn.Module，模仿了 torch.nn.Sequential 的功能\n",
    "class MySequential(nn.Module):\n",
    "    from collections import OrderedDict\n",
    "    #构造函数\n",
    "    def __init__(self, *args):\n",
    "        super(MySequential, self).__init__()\n",
    "        if len(args) == 1 and isinstance(args[0], OrderedDict): # 如果只传入一个元素，传⼊的是⼀个OrderedDict\n",
    "            for key, module in args[0].items():\n",
    "                self.add_module(key, module) # add_module⽅法会将module添加进self._modules(⼀个OrderedDict)\n",
    "        else: # 传⼊的是⼀些Module\n",
    "            for idx, module in enumerate(args):\n",
    "                self.add_module(str(idx), module)\n",
    "    def forward(self, input):\n",
    "        # self._modules返回⼀个 OrderedDict，保证会按照成员添加时的顺序遍历成\n",
    "        for module in self._modules.values():\n",
    "            input = module(input)\n",
    "        return input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0482c8e5-d453-4a34-bd69-950fbf36d740",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MySequential(\n",
      "  (0): Linear(in_features=784, out_features=256, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=256, out_features=10, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0739, -0.0464,  0.0137, -0.0688, -0.2503,  0.0553,  0.0869, -0.0437,\n",
       "          0.1713, -0.0028],\n",
       "        [-0.1592,  0.0719,  0.0582, -0.0485, -0.1850, -0.1617,  0.0920,  0.0119,\n",
       "          0.2284,  0.0231]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = MySequential(\n",
    "    nn.Linear(784, 256),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(256, 10), \n",
    "    )\n",
    "print(net)\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50071f86-9d1a-407e-bb3e-74c55616ed40",
   "metadata": {},
   "source": [
    "ModuleList 类"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "249594fe-ee75-4611-a4af-b18f92cce12a",
   "metadata": {},
   "source": [
    "ModuleList接收一个子模块的列表输入，可以类似List进行append与extend等操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "eb94173c-4dc5-4b61-b711-42fae46bdfeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear(in_features=256, out_features=10, bias=True)\n",
      "ModuleList(\n",
      "  (0): Linear(in_features=784, out_features=256, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=256, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = nn.ModuleList([nn.Linear(784, 256), nn.ReLU()])\n",
    "net.append(nn.Linear(256, 10)) # # 类似List的append操作\n",
    "print(net[-1]) # 类似List的索引访问\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ae207db-4c5a-402e-b659-b2f04c0fd174",
   "metadata": {},
   "source": [
    "ModuleDict 类"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cdf6cf1-e413-4033-bcf1-dd8638dfa895",
   "metadata": {},
   "source": [
    "ModuleDict 接收⼀个⼦模块的字典作为输⼊, 然后也可以类似字典那样进⾏添加访问操作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d203a269-0d7e-4130-a76e-e8edd4248df6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear(in_features=784, out_features=256, bias=True)\n",
      "Linear(in_features=256, out_features=10, bias=True)\n",
      "ModuleDict(\n",
      "  (linear): Linear(in_features=784, out_features=256, bias=True)\n",
      "  (act): ReLU()\n",
      "  (output): Linear(in_features=256, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = nn.ModuleDict({\n",
    "    'linear': nn.Linear(784, 256),\n",
    "    'act': nn.ReLU(),\n",
    "    })\n",
    "net['output'] = nn.Linear(256, 10) # 添加\n",
    "print(net['linear']) # 访问\n",
    "print(net.output)\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c87dab98-959a-44ee-8a0d-c0e81e5b04a2",
   "metadata": {},
   "source": [
    "构造复杂的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cd4852a9-d07e-46f8-8287-733a11d3b7bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FancyMLP(nn.Module):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(FancyMLP, self).__init__(**kwargs)\n",
    "        \n",
    "        self.rand_weight = torch.rand((20, 20), requires_grad=False) # 不可训练参数（常数参数）\n",
    "        self.linear = nn.Linear(20, 20)\n",
    "    def forward(self, x):\n",
    "        x = self.linear(x)\n",
    "        # 使⽤创建的常数参数，以及nn.functional中的relu函数和mm函数\n",
    "        x = nn.functional.relu(torch.mm(x, self.rand_weight.data) + 1)\n",
    "        \n",
    "        # 复⽤全连接层。等价于两个全连接层共享参数\n",
    "        #将经过 ReLU 激活后的 x 传递到另一个全连接层 self.linear 中，这样做的效果是复用了全连接层（该层的权重是共享的）\n",
    "        x = self.linear(x)\n",
    "        # 控制流，这⾥我们需要调⽤item函数来返回标量进⾏⽐较\n",
    "        while x.norm().item() > 1:\n",
    "            x /= 2\n",
    "        if x.norm().item() < 0.8:\n",
    "            x *= 10\n",
    "        return x.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e2640647-5f5b-42fe-9400-8f372e274501",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FancyMLP(\n",
      "  (linear): Linear(in_features=20, out_features=20, bias=True)\n",
      ")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(-0.3616, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = torch.rand(2, 20)\n",
    "net = FancyMLP()\n",
    "print(net)\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42abb19b-a68f-4b2a-9dad-b71133ca8188",
   "metadata": {},
   "source": [
    "在 FancyMLP 的前向传播过程中，模型不仅执行线性变换和激活函数，还根据输入的情况对数据进行了进一步的缩放和调整。具体来说：\n",
    "\n",
    "while 循环：根据 x 的范数（模长）动态调整输入的大小。\n",
    "if 判断：根据 x 的范数动态调整输入，保证其不小于一定的值。\n",
    "这种做法的目的是 动态调节数据的规模，使其在每次前向传播过程中都满足一定的约束条件，进而影响模型的表现。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcaec3e7-6059-4d47-8bb9-e6eeaa8be47c",
   "metadata": {},
   "source": [
    "这种做法的目的是控制X的大小，防止在模型中变大从而导致训练模型发散。过大的数值可能会导致梯度计算不稳定\n",
    "稳定性：通过在 while 循环中对数据进行缩放，避免了训练过程中出现过大或过小的数值，避免了数值不稳定的问题。通过动态调整输入数据的大小，模型可以保持稳定的训练过程。\n",
    "\n",
    "自适应：使用 if 判断语句可以让模型在输入的不同情况下做出不同的反应。例如，某些输入数据的范数可能比较小，放大它们有助于避免数据过小影响后续计算。通过这种自适应行为，模型可以更灵活地应对各种不同的数据分布。\n",
    "\n",
    "避免极端值：控制流可以帮助避免数值过大或过小的情况，这对于梯度更新过程至关重要。尤其在使用优化算法时，过大的数值可能会导致梯度爆炸，而过小的数值可能导致梯度消失。通过动态调整，模型避免了这些问题。\n",
    "\n",
    "探索非线性行为：传统的神经网络通常采用固定的权重和偏置进行训练，控制流的引入使得模型行为变得更加非线性。它允许模型在前向传播中进行动态计算，这使得模型在某些任务中可能具有更强的拟合能力。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0ab77c3e-7867-470f-bb0c-22812a2fe82a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): NestMLP(\n",
      "    (net): Sequential(\n",
      "      (0): Linear(in_features=40, out_features=30, bias=True)\n",
      "      (1): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (1): Linear(in_features=30, out_features=20, bias=True)\n",
      "  (2): FancyMLP(\n",
      "    (linear): Linear(in_features=20, out_features=20, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(0.2755, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class NestMLP(nn.Module):\n",
    "    def __init__(self, **kwargs):\n",
    "        super(NestMLP, self).__init__(**kwargs)\n",
    "        self.net = nn.Sequential(nn.Linear(40, 30), nn.ReLU()) \n",
    "    def forward(self, x):\n",
    "        return self.net(x)\n",
    "        \n",
    "net = nn.Sequential(NestMLP(), nn.Linear(30, 20), FancyMLP())\n",
    "X = torch.rand(2, 40)\n",
    "print(net)\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7c1caae-6693-4abd-94e2-ec2d733b4ed1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch]",
   "language": "python",
   "name": "deep_learning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
